{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.transforms import RandAugment\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "import time\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "from NonFcaNet import *\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)\n",
    " ## 修改模型\n",
    "net = fcanet(drop = 0.1).to(device)\n",
    "num_epochs=50\n",
    " # 定义损失函数和优化器\n",
    "trainloader,testloader = get_data_loader()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)\n",
    "scheduler = lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.1)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 0/49 轮训练\n",
      "----------\n",
      "train 损失: 4.0878 Top-1 准确率: 0.0811 Top-5 准确率: 0.2511\n",
      "val 损失: 3.9160 Top-1 准确率: 0.0911 Top-5 准确率: 0.2909\n",
      "\n",
      "第 1/49 轮训练\n",
      "----------\n",
      "train 损失: 3.5209 Top-1 准确率: 0.1713 Top-5 准确率: 0.4259\n",
      "val 损失: 3.4054 Top-1 准确率: 0.1850 Top-5 准确率: 0.4415\n",
      "\n",
      "第 2/49 轮训练\n",
      "----------\n",
      "train 损失: 3.1074 Top-1 准确率: 0.2505 Top-5 准确率: 0.5353\n",
      "val 损失: 3.0997 Top-1 准确率: 0.2236 Top-5 准确率: 0.5313\n",
      "\n",
      "第 3/49 轮训练\n",
      "----------\n",
      "train 损失: 2.8083 Top-1 准确率: 0.3064 Top-5 准确率: 0.6055\n",
      "val 损失: 2.7445 Top-1 准确率: 0.2972 Top-5 准确率: 0.6180\n",
      "\n",
      "第 4/49 轮训练\n",
      "----------\n",
      "train 损失: 2.5855 Top-1 准确率: 0.3567 Top-5 准确率: 0.6507\n",
      "val 损失: 2.4079 Top-1 准确率: 0.3677 Top-5 准确率: 0.6803\n",
      "\n",
      "第 5/49 轮训练\n",
      "----------\n",
      "train 损失: 2.3959 Top-1 准确率: 0.4025 Top-5 准确率: 0.6896\n",
      "val 损失: 2.2316 Top-1 准确率: 0.4064 Top-5 准确率: 0.7230\n",
      "\n",
      "第 6/49 轮训练\n",
      "----------\n",
      "train 损失: 2.2492 Top-1 准确率: 0.4338 Top-5 准确率: 0.7175\n",
      "val 损失: 2.1364 Top-1 准确率: 0.4393 Top-5 准确率: 0.7400\n",
      "\n",
      "第 7/49 轮训练\n",
      "----------\n",
      "train 损失: 2.1273 Top-1 准确率: 0.4600 Top-5 准确率: 0.7409\n",
      "val 损失: 2.1250 Top-1 准确率: 0.4335 Top-5 准确率: 0.7482\n",
      "\n",
      "第 8/49 轮训练\n",
      "----------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m model,(train_losses, train_top1_accs, train_top5_accs, val_losses, val_top1_accs, val_top5_accs) \u001b[38;5;241m=\u001b[39m \u001b[43mtrain_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnet\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtrainloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtestloader\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcriterion\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscheduler\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_epochs\u001b[49m\u001b[43m)\u001b[49m  \n",
      "File \u001b[0;32m~/deep-learning/FuTong/发起总攻/utils.py:163\u001b[0m, in \u001b[0;36mtrain_model\u001b[0;34m(model, trainloader, device, testloader, criterion, optimizer, scheduler, num_epochs)\u001b[0m\n\u001b[1;32m    160\u001b[0m         optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m    162\u001b[0m \u001b[38;5;66;03m# 统计\u001b[39;00m\n\u001b[0;32m--> 163\u001b[0m running_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m*\u001b[39m inputs\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m    164\u001b[0m running_corrects \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m top1_correct\n\u001b[1;32m    165\u001b[0m running_top1_corrects \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m top1_correct\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model,(train_losses, train_top1_accs, train_top5_accs, val_losses, val_top1_accs, val_top5_accs) = train_model(net,trainloader,device,testloader, criterion, optimizer, scheduler, num_epochs)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_cpu_numpy(tensor):\n",
    "    return tensor.cpu().numpy()\n",
    "def to_cpu(list_):\n",
    "    if isinstance(list_, list):\n",
    "        return list(map(to_cpu_numpy,list_))\n",
    "    else:\n",
    "        return to_cpu_numpy(list_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_training_results(train_losses, to_cpu(train_top1_accs), to_cpu(train_top5_accs), val_losses, to_cpu(val_top1_accs), to_cpu(val_top5_accs))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 0/9 轮训练\n",
      "----------\n",
      "train 损失: 1.0620 Top-1 准确率: 0.7324 Top-5 准确率: 0.9468\n",
      "val 损失: 0.9733 Top-1 准确率: 0.7349 Top-5 准确率: 0.9241\n",
      "\n",
      "第 1/9 轮训练\n",
      "----------\n",
      "train 损失: 1.0028 Top-1 准确率: 0.7459 Top-5 准确率: 0.9599\n",
      "val 损失: 0.9596 Top-1 准确率: 0.7383 Top-5 准确率: 0.9252\n",
      "\n",
      "第 2/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9782 Top-1 准确率: 0.7504 Top-5 准确率: 0.9640\n",
      "val 损失: 0.9526 Top-1 准确率: 0.7412 Top-5 准确率: 0.9262\n",
      "\n",
      "第 3/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9559 Top-1 准确率: 0.7557 Top-5 准确率: 0.9679\n",
      "val 损失: 0.9316 Top-1 准确率: 0.7485 Top-5 准确率: 0.9301\n",
      "\n",
      "第 4/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9408 Top-1 准确率: 0.7592 Top-5 准确率: 0.9712\n",
      "val 损失: 0.9461 Top-1 准确率: 0.7447 Top-5 准确率: 0.9286\n",
      "\n",
      "第 5/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9439 Top-1 准确率: 0.7568 Top-5 准确率: 0.9732\n",
      "val 损失: 0.9388 Top-1 准确率: 0.7463 Top-5 准确率: 0.9284\n",
      "\n",
      "第 6/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9196 Top-1 准确率: 0.7632 Top-5 准确率: 0.9740\n",
      "val 损失: 0.9633 Top-1 准确率: 0.7412 Top-5 准确率: 0.9272\n",
      "\n",
      "第 7/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9159 Top-1 准确率: 0.7628 Top-5 准确率: 0.9758\n",
      "val 损失: 0.9461 Top-1 准确率: 0.7429 Top-5 准确率: 0.9283\n",
      "\n",
      "第 8/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9181 Top-1 准确率: 0.7626 Top-5 准确率: 0.9765\n",
      "val 损失: 0.9518 Top-1 准确率: 0.7426 Top-5 准确率: 0.9278\n",
      "\n",
      "第 9/9 轮训练\n",
      "----------\n",
      "train 损失: 0.9076 Top-1 准确率: 0.7639 Top-5 准确率: 0.9782\n",
      "val 损失: 0.9446 Top-1 准确率: 0.7464 Top-5 准确率: 0.9271\n",
      "\n",
      "训练完成，耗时 6m 35s\n",
      "最佳验证集准确率: 0.748500\n"
     ]
    }
   ],
   "source": [
    "model2,(train_losses2, train_top1_accs2, train_top5_accs2, val_losses2, val_top1_accs2, val_top5_accs2) = train_model(model,trainloader,device,testloader, criterion, optimizer, scheduler, num_epochs = 10)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 0/29 轮训练\n",
      "----------\n",
      "train 损失: 0.9546 Top-1 准确率: 0.7545 Top-5 准确率: 0.9690\n",
      "val 损失: 0.9359 Top-1 准确率: 0.7464 Top-5 准确率: 0.9290\n",
      "\n",
      "第 1/29 轮训练\n",
      "----------\n",
      "train 损失: 0.9525 Top-1 准确率: 0.7550 Top-5 准确率: 0.9719\n",
      "val 损失: 0.9363 Top-1 准确率: 0.7467 Top-5 准确率: 0.9295\n",
      "\n",
      "第 2/29 轮训练\n",
      "----------\n",
      "train 损失: 0.9217 Top-1 准确率: 0.7626 Top-5 准确率: 0.9740\n",
      "val 损失: 0.9395 Top-1 准确率: 0.7467 Top-5 准确率: 0.9294\n",
      "\n",
      "第 3/29 轮训练\n",
      "----------\n",
      "train 损失: 0.9126 Top-1 准确率: 0.7647 Top-5 准确率: 0.9746\n",
      "val 损失: 0.9408 Top-1 准确率: 0.7461 Top-5 准确率: 0.9285\n",
      "\n",
      "第 4/29 轮训练\n",
      "----------\n",
      "train 损失: 0.9196 Top-1 准确率: 0.7613 Top-5 准确率: 0.9766\n",
      "val 损失: 0.9449 Top-1 准确率: 0.7456 Top-5 准确率: 0.9297\n",
      "\n",
      "第 5/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8961 Top-1 准确率: 0.7682 Top-5 准确率: 0.9778\n",
      "val 损失: 0.9288 Top-1 准确率: 0.7474 Top-5 准确率: 0.9298\n",
      "\n",
      "第 6/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8941 Top-1 准确率: 0.7680 Top-5 准确率: 0.9797\n",
      "val 损失: 0.9266 Top-1 准确率: 0.7504 Top-5 准确率: 0.9309\n",
      "\n",
      "第 7/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8886 Top-1 准确率: 0.7687 Top-5 准确率: 0.9787\n",
      "val 损失: 0.9423 Top-1 准确率: 0.7449 Top-5 准确率: 0.9289\n",
      "\n",
      "第 8/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8888 Top-1 准确率: 0.7686 Top-5 准确率: 0.9810\n",
      "val 损失: 0.9418 Top-1 准确率: 0.7473 Top-5 准确率: 0.9296\n",
      "\n",
      "第 9/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8828 Top-1 准确率: 0.7700 Top-5 准确率: 0.9805\n",
      "val 损失: 0.9418 Top-1 准确率: 0.7474 Top-5 准确率: 0.9303\n",
      "\n",
      "第 10/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8748 Top-1 准确率: 0.7704 Top-5 准确率: 0.9823\n",
      "val 损失: 0.9509 Top-1 准确率: 0.7466 Top-5 准确率: 0.9291\n",
      "\n",
      "第 11/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8693 Top-1 准确率: 0.7723 Top-5 准确率: 0.9828\n",
      "val 损失: 0.9236 Top-1 准确率: 0.7550 Top-5 准确率: 0.9331\n",
      "\n",
      "第 12/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8696 Top-1 准确率: 0.7714 Top-5 准确率: 0.9833\n",
      "val 损失: 0.9516 Top-1 准确率: 0.7487 Top-5 准确率: 0.9282\n",
      "\n",
      "第 13/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8559 Top-1 准确率: 0.7758 Top-5 准确率: 0.9842\n",
      "val 损失: 0.9420 Top-1 准确率: 0.7489 Top-5 准确率: 0.9288\n",
      "\n",
      "第 14/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8558 Top-1 准确率: 0.7747 Top-5 准确率: 0.9840\n",
      "val 损失: 0.9451 Top-1 准确率: 0.7503 Top-5 准确率: 0.9300\n",
      "\n",
      "第 15/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8477 Top-1 准确率: 0.7767 Top-5 准确率: 0.9851\n",
      "val 损失: 0.9533 Top-1 准确率: 0.7479 Top-5 准确率: 0.9307\n",
      "\n",
      "第 16/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8477 Top-1 准确率: 0.7773 Top-5 准确率: 0.9856\n",
      "val 损失: 0.9480 Top-1 准确率: 0.7502 Top-5 准确率: 0.9295\n",
      "\n",
      "第 17/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8601 Top-1 准确率: 0.7729 Top-5 准确率: 0.9856\n",
      "val 损失: 0.9595 Top-1 准确率: 0.7489 Top-5 准确率: 0.9282\n",
      "\n",
      "第 18/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8432 Top-1 准确率: 0.7768 Top-5 准确率: 0.9863\n",
      "val 损失: 0.9530 Top-1 准确率: 0.7536 Top-5 准确率: 0.9293\n",
      "\n",
      "第 19/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8407 Top-1 准确率: 0.7767 Top-5 准确率: 0.9875\n",
      "val 损失: 0.9473 Top-1 准确率: 0.7507 Top-5 准确率: 0.9285\n",
      "\n",
      "第 20/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8350 Top-1 准确率: 0.7786 Top-5 准确率: 0.9867\n",
      "val 损失: 0.9454 Top-1 准确率: 0.7526 Top-5 准确率: 0.9302\n",
      "\n",
      "第 21/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8300 Top-1 准确率: 0.7806 Top-5 准确率: 0.9876\n",
      "val 损失: 0.9463 Top-1 准确率: 0.7538 Top-5 准确率: 0.9296\n",
      "\n",
      "第 22/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8294 Top-1 准确率: 0.7790 Top-5 准确率: 0.9883\n",
      "val 损失: 0.9534 Top-1 准确率: 0.7528 Top-5 准确率: 0.9279\n",
      "\n",
      "第 23/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8252 Top-1 准确率: 0.7804 Top-5 准确率: 0.9886\n",
      "val 损失: 0.9584 Top-1 准确率: 0.7525 Top-5 准确率: 0.9301\n",
      "\n",
      "第 24/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8244 Top-1 准确率: 0.7795 Top-5 准确率: 0.9889\n",
      "val 损失: 0.9577 Top-1 准确率: 0.7520 Top-5 准确率: 0.9268\n",
      "\n",
      "第 25/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8117 Top-1 准确率: 0.7844 Top-5 准确率: 0.9890\n",
      "val 损失: 0.9643 Top-1 准确率: 0.7511 Top-5 准确率: 0.9277\n",
      "\n",
      "第 26/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8115 Top-1 准确率: 0.7832 Top-5 准确率: 0.9894\n",
      "val 损失: 0.9662 Top-1 准确率: 0.7522 Top-5 准确率: 0.9298\n",
      "\n",
      "第 27/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8187 Top-1 准确率: 0.7811 Top-5 准确率: 0.9897\n",
      "val 损失: 0.9549 Top-1 准确率: 0.7548 Top-5 准确率: 0.9297\n",
      "\n",
      "第 28/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8099 Top-1 准确率: 0.7833 Top-5 准确率: 0.9901\n",
      "val 损失: 0.9763 Top-1 准确率: 0.7525 Top-5 准确率: 0.9245\n",
      "\n",
      "第 29/29 轮训练\n",
      "----------\n",
      "train 损失: 0.8080 Top-1 准确率: 0.7825 Top-5 准确率: 0.9904\n",
      "val 损失: 0.9498 Top-1 准确率: 0.7533 Top-5 准确率: 0.9285\n",
      "\n",
      "训练完成，耗时 19m 53s\n",
      "最佳验证集准确率: 0.755000\n"
     ]
    }
   ],
   "source": [
    "model3,(train_losses3, train_top1_accs3, train_top5_accs3, val_losses3, val_top1_accs3, val_top5_accs3) = train_model(model2,trainloader,device,testloader, criterion, optimizer, scheduler, num_epochs = 30)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model3, 'fixed90epoch_highway+02drop.pth')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里的隐藏层是1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
